{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Agent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting Started\n",
    "\n",
    "This sample demonstrates how to evaluate Azure AI Agent\n",
    "Before running the sample:\n",
    "```bash\n",
    "pip install azure-ai-projects azure-identity azure-ai-evaluation\n",
    "```\n",
    "Set these environment variables with your own values:\n",
    "1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n",
    "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the AI model, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n",
    "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n",
    "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n",
    "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n",
    "6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n",
    "7) **PROJECT_NAME** - Azure AI Project Name\n",
    "8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, json\n",
    "import pandas as pd\n",
    "import time\n",
    "from azure.ai.projects import AIProjectClient\n",
    "from azure.identity import DefaultAzureCredential\n",
    "from user_functions import user_functions\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "from azure.ai.projects import __version__ as projects_version\n",
    "from packaging.version import Version\n",
    "# some dependencies have been updated with breaking changes -- indicates whether to use updated models and APIs or not\n",
    "updated_agents = Version(projects_version) > Version(\"1.0.0b10\") or projects_version.startswith(\"1.0.0a\")\n",
    "\n",
    "if updated_agents:\n",
    "    from azure.ai.agents.models import FunctionTool, ToolSet\n",
    "    project_client = AIProjectClient(\n",
    "        endpoint=os.environ[\"PROJECT_ENDPOINT\"],\n",
    "        credential=DefaultAzureCredential(),\n",
    "        api_version=\"latest\"\n",
    "    )\n",
    "else:\n",
    "    from azure.ai.projects.models import FunctionTool, ToolSet\n",
    "    project_client = AIProjectClient.from_connection_string(\n",
    "        credential=DefaultAzureCredential(),\n",
    "        conn_str=os.environ[\"PROJECT_CONNECTION_STRING\"],\n",
    "    )\n",
    "\n",
    "AGENT_NAME = \"Seattle Tourist Assistant PrP\"\n",
    "\n",
    "# Adding Tools to be used by Agent \n",
    "functions = FunctionTool(user_functions)\n",
    "\n",
    "toolset = ToolSet()\n",
    "toolset.add(functions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = project_client.agents.create_agent(\n",
    "    model=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n",
    "    name=AGENT_NAME,\n",
    "    instructions=\"You are a helpful assistant\",\n",
    "    toolset=toolset\n",
    ")\n",
    "\n",
    "print(f\"Created agent, ID: {agent.id}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Thread"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if updated_agents:\n",
    "    thread = project_client.agents.threads.create()\n",
    "else:\n",
    "    thread = project_client.agents.create_thread()\n",
    "print(f\"Created thread, ID: {thread.id}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conversation with Agent\n",
    "Use below cells to have conversation with the agent\n",
    "- `Create Message[1]`\n",
    "- `Execute[2]`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Message[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create message to thread\n",
    "\n",
    "MESSAGE = \"Can you send me an email with weather information for Seattle?\"\n",
    "\n",
    "if updated_agents:\n",
    "    message = project_client.agents.messages.create(\n",
    "        thread_id=thread.id,\n",
    "        role=\"user\",\n",
    "        content=MESSAGE,\n",
    "    )        \n",
    "else:\n",
    "    message = project_client.agents.create_message(\n",
    "        thread_id=thread.id,\n",
    "        role=\"user\",\n",
    "        content=MESSAGE,\n",
    "    )\n",
    "    \n",
    "print(f\"Created message, ID: {message.id}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Execute[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "if updated_agents:\n",
    "    from azure.ai.agents.models import (\n",
    "        FunctionTool,\n",
    "        ListSortOrder,\n",
    "        RequiredFunctionToolCall,\n",
    "        SubmitToolOutputsAction,\n",
    "        ToolOutput,\n",
    "    )\n",
    "    run = project_client.agents.runs.create(thread_id=thread.id, agent_id=agent.id)\n",
    "    \n",
    "    while run.status in [\"queued\", \"in_progress\", \"requires_action\"]:\n",
    "        time.sleep(1)\n",
    "        run = project_client.agents.runs.get(thread_id=thread.id, run_id=run.id)\n",
    "\n",
    "        if run.status == \"requires_action\" and isinstance(run.required_action, SubmitToolOutputsAction):\n",
    "            tool_calls = run.required_action.submit_tool_outputs.tool_calls\n",
    "            if not tool_calls:\n",
    "                print(\"No tool calls provided - cancelling run\")\n",
    "                project_client.agents.runs.cancel(thread_id=thread.id, run_id=run.id)\n",
    "                break\n",
    "\n",
    "            tool_outputs = []\n",
    "            for tool_call in tool_calls:\n",
    "                if isinstance(tool_call, RequiredFunctionToolCall):\n",
    "                    try:\n",
    "                        print(f\"Executing tool call: {tool_call}\")\n",
    "                        output = functions.execute(tool_call)\n",
    "                        tool_outputs.append(\n",
    "                            ToolOutput(\n",
    "                                tool_call_id=tool_call.id,\n",
    "                                output=output,\n",
    "                            )\n",
    "                        )\n",
    "                    except Exception as e:\n",
    "                        print(f\"Error executing tool_call {tool_call.id}: {e}\")\n",
    "\n",
    "            print(f\"Tool outputs: {tool_outputs}\")\n",
    "            if tool_outputs:\n",
    "                project_client.agents.runs.submit_tool_outputs(thread_id=thread.id, run_id=run.id, tool_outputs=tool_outputs)\n",
    "    print(f\"Run status: {run.status}\")\n",
    "\n",
    "else:\n",
    "    from azure.ai.projects.models import (\n",
    "        FunctionTool,\n",
    "        ListSortOrder,\n",
    "        RequiredFunctionToolCall,\n",
    "        SubmitToolOutputsAction,\n",
    "        ToolOutput,\n",
    "    )\n",
    "    run = project_client.agents.create_run(thread_id=thread.id, agent_id=agent.id)\n",
    "    while run.status in [\"queued\", \"in_progress\", \"requires_action\"]:\n",
    "        time.sleep(1)\n",
    "        run = project_client.agents.get_run(thread_id=thread.id, run_id=run.id)\n",
    "\n",
    "        if run.status == \"requires_action\" and isinstance(run.required_action, SubmitToolOutputsAction):\n",
    "            tool_calls = run.required_action.submit_tool_outputs.tool_calls\n",
    "            if not tool_calls:\n",
    "                print(\"No tool calls provided - cancelling run\")\n",
    "                project_client.agents.cancel_run(thread_id=thread.id, run_id=run.id)\n",
    "                break\n",
    "\n",
    "            tool_outputs = []\n",
    "            for tool_call in tool_calls:\n",
    "                if isinstance(tool_call, RequiredFunctionToolCall):\n",
    "                    try:\n",
    "                        print(f\"Executing tool call: {tool_call}\")\n",
    "                        output = functions.execute(tool_call)\n",
    "                        tool_outputs.append(\n",
    "                            ToolOutput(\n",
    "                                tool_call_id=tool_call.id,\n",
    "                                output=output,\n",
    "                            )\n",
    "                        )\n",
    "                    except Exception as e:\n",
    "                        print(f\"Error executing tool_call {tool_call.id}: {e}\")\n",
    "\n",
    "            print(f\"Tool outputs: {tool_outputs}\")\n",
    "            if tool_outputs:\n",
    "                project_client.agents.submit_tool_outputs_to_run(thread_id=thread.id, run_id=run.id, tool_outputs=tool_outputs)\n",
    "    print(f\"Run status: {run.status}\")\n",
    "\n",
    "\n",
    "print(f\"Run finished with status: {run.status}\")\n",
    "\n",
    "if run.status == \"failed\":\n",
    "    print(f\"Run failed: {run.last_error}\")\n",
    "\n",
    "print(f\"Run ID: {run.id}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### List Messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if updated_agents:\n",
    "    for message in project_client.agents.messages.list(thread.id, order=\"asc\"):\n",
    "        print(f\"Role: {message.role}\")\n",
    "        print(f\"Content: {message.content[0].text.value}\")\n",
    "        print(\"-\" * 40)\n",
    "else:\n",
    "    for message in project_client.agents.list_messages(thread.id, order=\"asc\").data:\n",
    "        print(f\"Role: {message.role}\")\n",
    "        print(f\"Content: {message.content[0].text.value}\")\n",
    "        print(\"-\" * 40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get data from agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from azure.ai.evaluation import AIAgentConverter\n",
    "\n",
    "# Initialize the converter that will be backed by the project.\n",
    "converter = AIAgentConverter(project_client)\n",
    "\n",
    "thread_id = thread.id\n",
    "run_id = run.id\n",
    "\n",
    "converted_data = converter.convert(thread_id=thread_id, run_id=run_id)\n",
    "print(json.dumps(converted_data, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the converted data to a JSONL file\n",
    "\n",
    "file_name = \"evaluation_data.jsonl\"\n",
    "evaluation_data = converter.prepare_evaluation_data(thread_ids=thread.id, filename=file_name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_dotenv()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setting up evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azure.ai.evaluation import ToolCallAccuracyEvaluator , AzureOpenAIModelConfiguration, IntentResolutionEvaluator, TaskAdherenceEvaluator\n",
    "from pprint import pprint\n",
    "\n",
    "model_config = AzureOpenAIModelConfiguration(\n",
    "    azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n",
    "    api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n",
    "    api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n",
    "    azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n",
    ")\n",
    "# Needed to use content safety evaluators\n",
    "azure_ai_project={\n",
    "    \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n",
    "    \"project_name\": os.environ[\"PROJECT_NAME\"],\n",
    "    \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n",
    "}\n",
    "\n",
    "tool_call_accuracy = ToolCallAccuracyEvaluator(model_config=model_config)\n",
    "intent_resolution = IntentResolutionEvaluator(model_config=model_config)\n",
    "task_adherence = TaskAdherenceEvaluator(model_config=model_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tool_call_accuracy(query=converted_data['query'], response=converted_data['response'], tool_definitions=converted_data['tool_definitions'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run Evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azure.ai.evaluation import evaluate\n",
    "\n",
    "response = evaluate(\n",
    "    data=file_name,\n",
    "    evaluators={\n",
    "        \"tool_call_accuracy\": tool_call_accuracy,\n",
    "        \"intent_resolution\": intent_resolution,\n",
    "        \"task_adherence\": task_adherence\n",
    "    },\n",
    "    azure_ai_project={\n",
    "        \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n",
    "        \"project_name\": os.environ[\"PROJECT_NAME\"],\n",
    "        \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n",
    "    }\n",
    ")\n",
    "pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pprint(response)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
